{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Neural Language Models\n",
    "Status of Notebook: Work in Progress\n",
    "\n",
    "Reference: https://www.jmlr.org/papers/volume3/bengio03a/bengio03a.pdf\n",
    "\n",
    "Dynet Version: https://github.com/neubig/nn4nlp-code/blob/master/02-lm/nn-lm.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import torch\n",
    "import random\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import math\n",
    "import time\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Download the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# uncomment to download the datasets\n",
    "#!wget https://raw.githubusercontent.com/neubig/nn4nlp-code/master/data/ptb/test.txt\n",
    "#!wget https://raw.githubusercontent.com/neubig/nn4nlp-code/master/data/ptb/train.txt\n",
    "#!wget https://raw.githubusercontent.com/neubig/nn4nlp-code/master/data/ptb/valid.txt"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process the Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# function to read in data, pro=ess each line and split columns by \" ||| \"\n",
    "def read_data(filename):\n",
    "    data = []\n",
    "    with open(filename, \"r\") as f:\n",
    "        for line in f:\n",
    "            line = line.strip().split(\" \")\n",
    "            data.append(line)\n",
    "    return data\n",
    "\n",
    "# read the data\n",
    "train_data = read_data('data/ptb/train.txt')\n",
    "val_data = read_data('data/ptb/valid.txt')\n",
    "\n",
    "# creating the word and tag indices and special tokens\n",
    "word_to_index = {}\n",
    "index_to_word = {}\n",
    "word_to_index[\"<s>\"] = len(word_to_index)\n",
    "index_to_word[len(word_to_index)-1] = \"<s>\"\n",
    "word_to_index[\"<unk>\"] = len(word_to_index) # add <UNK> to dictionary\n",
    "index_to_word[len(word_to_index)-1] = \"<unk>\"\n",
    "\n",
    "# create word to index dictionary and tag to index dictionary from data\n",
    "def create_dict(data, check_unk=False):\n",
    "    for line in data:\n",
    "        for word in line:\n",
    "            if check_unk == False:\n",
    "                if word not in word_to_index:\n",
    "                    word_to_index[word] = len(word_to_index)\n",
    "                    index_to_word[len(word_to_index)-1] = word\n",
    "            \n",
    "            # has no effect because data already comes with <unk>\n",
    "            # should work with data without <unk> already processed\n",
    "            else: \n",
    "                if word not in word_to_index:\n",
    "                    word_to_index[word] = word_to_index[\"<unk>\"]\n",
    "                    index_to_word[len(word_to_index)-1] = word\n",
    "\n",
    "create_dict(train_data)\n",
    "create_dict(val_data, check_unk=True)\n",
    "\n",
    "# create word and tag tensors from data\n",
    "def create_tensor(data):\n",
    "    for line in data:\n",
    "        yield([word_to_index[word] for word in line])\n",
    "\n",
    "train_data = list(create_tensor(train_data))\n",
    "val_data = list(create_tensor(val_data))\n",
    "\n",
    "number_of_words = len(word_to_index)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "In our implementation we are using batched training. There are a few differences from the original implementation found [here](https://github.com/neubig/nn4nlp-code/blob/master/02-lm/loglin-lm.py). "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Define the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "## define the model\n",
    "\n",
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'\n",
    "\n",
    "N = 2 # length of the n-gram\n",
    "EMB_SIZE = 128 # size of the embedding\n",
    "HID_SIZE = 128 # size of the hidden layer\n",
    "\n",
    "# Neural LM\n",
    "class NeuralLM(nn.Module):\n",
    "    def __init__(self, number_of_words, ngram_length, EMB_SIZE, HID_SIZE):\n",
    "        super(NeuralLM, self).__init__()\n",
    "\n",
    "        # embedding layer\n",
    "        self.embedding = nn.Embedding(number_of_words, EMB_SIZE)\n",
    "\n",
    "        # hidden layer\n",
    "        self.hidden = nn.Linear(EMB_SIZE * ngram_length, HID_SIZE)\n",
    "        # output layer\n",
    "        self.output = nn.Linear(HID_SIZE, number_of_words)\n",
    "\n",
    "    def forward(self, x):\n",
    "        embs = self.embedding(x)                        # Size: [batch_size x num_hist x emb_size]\n",
    "        embs = embs.view(embs.size(0), -1)              # Size: [batch_size x (num_hist*emb_size)]\n",
    "        h = torch.nn.functional.tanh(self.hidden(embs)) # Size: [batch_size x hid_size]\n",
    "        scores = self.output(h)                         # Size: batch_size x num_words\n",
    "        return scores"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Model Settings and Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = NeuralLM(number_of_words, N, EMB_SIZE, HID_SIZE)\n",
    "optimizer = torch.optim.Adam(model.parameters(), lr=0.1)\n",
    "criterion = torch.nn.CrossEntropyLoss()\n",
    "\n",
    "if torch.cuda.is_available():\n",
    "    model.to(device)\n",
    "\n",
    "# function to calculate the sentence loss\n",
    "def calc_sent_loss(sent):\n",
    "    S = word_to_index[\"<s>\"]\n",
    "    \n",
    "    # initial history is equal to end of sentence symbols\n",
    "    hist = [S] * N\n",
    "    \n",
    "    # collect all target and histories\n",
    "    all_targets = []\n",
    "    all_histories = []\n",
    "    \n",
    "    # step through the sentence, including the end of sentence token\n",
    "    for next_word in sent + [S]:\n",
    "        all_histories.append(list(hist))\n",
    "        all_targets.append(next_word)\n",
    "        hist = hist[1:] + [next_word]\n",
    "\n",
    "    logits = model(torch.LongTensor(all_histories).to(device))\n",
    "    loss = criterion(logits, torch.LongTensor(all_targets).to(device))\n",
    "\n",
    "    return loss\n",
    "\n",
    "MAX_LEN = 100\n",
    "# Function to generate a sentence\n",
    "def generate_sent():\n",
    "    S = word_to_index[\"<s>\"]\n",
    "    hist = [S] * N\n",
    "    sent = []\n",
    "    while True:\n",
    "        logits = model(torch.LongTensor([hist]).to(device))\n",
    "        p = torch.nn.functional.softmax(logits) # 1 x number_of_words\n",
    "        next_word = p.multinomial(num_samples=1).item()\n",
    "        if next_word == S or len(sent) == MAX_LEN:\n",
    "            break\n",
    "        sent.append(next_word)\n",
    "        hist = hist[1:] + [next_word]\n",
    "    return sent"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Train the Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 0: train loss/word=4.1802, ppl=65.3775\n",
      "iter 0: dev loss/word=4.4128, ppl=82.4961, time=1.26s\n",
      "in constitution physics which could counting suspect include be on\n",
      "dealers manufacturers plans commissions\n",
      "in constitution physics which could counting suspect include be on behalf he declares\n",
      "in constitution physics which could counting suspect include be and which and an for was designed on themes of weakness jobs n't be and which <unk> developed the sale such from about other objectives\n",
      "N have in prolonged damage\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/envs/nlp/lib/python3.7/site-packages/ipykernel_launcher.py:38: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 1: train loss/word=4.4307, ppl=83.9873\n",
      "iter 1: dev loss/word=4.5315, ppl=92.8970, time=1.27s\n",
      "two <unk> hours <unk> relations clark new an index big were medicine more 'm bank N in october this fall\n",
      "two this said its consumer puts for democratic futures the ringers out N note a day to get this affairs\n",
      "this time\n",
      "two this said its consumer puts for democratic futures the ringers out N note a top outstanding bank for $ candidates savings <unk> relationship in shopping for declared futures the ringers out N note a day to get this said its consumer puts to highlight this fall\n",
      "this time the\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 2: train loss/word=4.4670, ppl=87.0953\n",
      "iter 2: dev loss/word=4.5699, ppl=96.5306, time=1.28s\n",
      "there is by being it on the first was estimated for possible the experiment of those after the key\n",
      "there is by\n",
      "<unk> intensity excess $ co. spot N N N N N the tanker of those after in <unk> forced around who participated in <unk> forced around to $ N million wednesday\n",
      "there is by\n",
      "to one why N gallons iii bush from $ N million wednesday\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 3: train loss/word=4.4909, ppl=89.1985\n",
      "iter 3: dev loss/word=4.5530, ppl=94.9163, time=1.31s\n",
      "in of western actions it does about service pilots a the company costs there with chief executive retailing <unk> under and are will for which the department showed N N of stock funds profit as well a buildup and an interest has expects <unk> up in the friday-the-13th third\n",
      "in of <unk> and the products costs has about shareholders fidelity with agreed was it to a less and stock will okla. say the former economic to make\n",
      "in it does about service pilots a the company costs there with a five-year and then <unk> more\n",
      "in of western actions it does about service pilots at profit common runs has\n",
      "thus declined to comment\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 4: train loss/word=4.4966, ppl=89.7113\n",
      "iter 4: dev loss/word=4.6409, ppl=103.6412, time=1.28s\n",
      "the apparent centers groups by to reform\n",
      "are consumers too deep over that we do n't want to continue owning stocks we oct. in <unk>\n",
      "the apparent centers groups by the <unk> missile n't available five former corp to slow owning u.k. <unk>\n",
      "the apparent centers groups by the and centers but to limit owning investment\n",
      "the apparent centers groups by the to share mr. lehman attributed n't available five former corp to slow owning it was who i was coast to round owning revenue to specific another dramatic worked and financial announcements <unk> wo n't end anytime soon\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 5: train loss/word=4.5213, ppl=91.9530\n",
      "iter 5: dev loss/word=4.7837, ppl=119.5463, time=1.28s\n",
      "the other involving plant the commission value to consolidate several lawsuits <unk> senior the commission value to consolidate several lawsuits <unk> senior the commission value to consolidate several lawsuits in many other say central\n",
      "the other involving plant the commission value to consolidate several lawsuits <unk> senior the commission value to consolidate several lawsuits is for filled <unk> senior the commission value to consolidate several lawsuits in many other say central\n",
      "the other involving plant the commission groups\n",
      "the other involving plant the commission value of leading funds code growth channel grows aide market against her for clearance 's <unk> of the <unk> era of the <unk> era of the <unk> era of the <unk> era of the <unk> era to mr. a ratio process some air fares about rumors <unk> <unk> process the <unk> era of the <unk> era the <unk> era of the <unk> era of the <unk> era of the <unk> era of the <unk> era of the <unk> era of the <unk> era of the <unk> era to mr. the <unk> era to mr.\n",
      "the other involving plant the commission value bank leading some continental a senior the commission value to consolidate several lawsuits on fetch for violations <unk> senior the commission value to consolidate several lawsuits <unk> senior the commission value to consolidate several lawsuits <unk> senior the commission value in draw other time strict fire american express for two operations n't if debt only on behalf n't increase he also questioned for two operations n't if debt\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 6: train loss/word=4.5284, ppl=92.6074\n",
      "iter 6: dev loss/word=4.8860, ppl=132.4199, time=1.27s\n",
      "toyota have expressed recent durable attempt <unk> chief stocks spend notes\n",
      "toyota have expressed recent durable attempt <unk> development\n",
      "toyota have expressed recent durable attempt to\n",
      "toyota have expressed recent durable attempt <unk> development 's also <unk> concedes stocks could back average resolve by\n",
      "toyota of N <unk> occurred stocks less today <unk>\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 7: train loss/word=4.5339, ppl=93.1220\n",
      "iter 7: dev loss/word=4.9127, ppl=136.0103, time=1.28s\n",
      "the distributor environment wealth fleet mosbacher N N from turnover citing commitment place the partnership more than year <unk> <unk> new york $ N to N a share a year earlier assets at fairly insurance will open and assets <unk> subsidiaries by and mae <unk> and his wife by fannie mae N N from turnover citing commitment place the partnership more than a partial on <unk> changes by of companies and assets <unk> subsidiaries in a trading range N N from turnover citing commitment place the partnership more than year <unk> <unk> new york $ N to N to N\n",
      "the distributor environment wealth fleet mosbacher N N from turnover citing commitment place only will open to represent this week\n",
      "the distributor environment wealth fleet mosbacher N N from turnover citing commitment place only will open\n",
      "the distributor environment wealth fleet mosbacher N N from turnover citing commitment place the partnership more than year <unk> <unk> new york $ N to N N from turnover citing commitment place only will open and assets <unk> subsidiaries that not officials monday N N from turnover citing commitment place the partnership more other <unk> match by by mae <unk> and <unk> and his wife by fannie mae N N from turnover citing commitment place the partnership more than a partial of refusing and assets <unk> subsidiaries by and mae <unk> and his wife by fannie mae N N from\n",
      "the distributor environment wealth fleet mosbacher N N from turnover citing commitment place only will open to represent this week\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n",
      "--finished 20000 sentences\n",
      "--finished 25000 sentences\n",
      "--finished 30000 sentences\n",
      "--finished 35000 sentences\n",
      "--finished 40000 sentences\n",
      "iter 8: train loss/word=4.5402, ppl=93.7127\n",
      "iter 8: dev loss/word=4.9000, ppl=134.2900, time=1.28s\n",
      "barney any projections case for purchase\n",
      "barney any projections case for purchase about liability says fast financial profit would increase close sassy cie are into imports the will concern the work industry did doing to record other cash and the but section N <unk> N gorbachev games the continued mr. sohmer says <unk> for N N aided executive into attacking and increased risks <unk> where with ratings pitch democratic\n",
      "barney any projections case for purchase\n",
      "barney any projections case for purchase\n",
      "barney any projections case for purchase\n",
      "--finished 5000 sentences\n",
      "--finished 10000 sentences\n",
      "--finished 15000 sentences\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m/tmp/ipykernel_14352/404430955.py\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0msent_id\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0msent\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_data\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;31m# CHANGE to all train_data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m         \u001b[0mmy_loss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcalc_sent_loss\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msent\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m         \u001b[0mtrain_loss\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0mmy_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_14352/2289298869.py\u001b[0m in \u001b[0;36mcalc_sent_loss\u001b[0;34m(sent)\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0mhist\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhist\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m+\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mnext_word\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 25\u001b[0;31m     \u001b[0mlogits\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mmodel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLongTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_histories\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     26\u001b[0m     \u001b[0mloss\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlogits\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mLongTensor\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall_targets\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mto\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdevice\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/opt/conda/envs/nlp/lib/python3.7/site-packages/torch/nn/modules/module.py\u001b[0m in \u001b[0;36m_call_impl\u001b[0;34m(self, *input, **kwargs)\u001b[0m\n\u001b[1;32m   1108\u001b[0m         if not (self._backward_hooks or self._forward_hooks or self._forward_pre_hooks or _global_backward_hooks\n\u001b[1;32m   1109\u001b[0m                 or _global_forward_hooks or _global_forward_pre_hooks):\n\u001b[0;32m-> 1110\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mforward_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0minput\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1111\u001b[0m         \u001b[0;31m# Do not call functions when jit is used\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1112\u001b[0m         \u001b[0mfull_backward_hooks\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mnon_full_backward_hooks\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/tmp/ipykernel_14352/2217491764.py\u001b[0m in \u001b[0;36mforward\u001b[0;34m(self, x)\u001b[0m\n\u001b[1;32m     22\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mforward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m         \u001b[0membs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0membedding\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# [batch_size x num_hist x emb_size]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 24\u001b[0;31m         \u001b[0membs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0membs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mview\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# [batch_size x (num_hist*emb_size)]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     25\u001b[0m         \u001b[0mh\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfunctional\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtanh\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhidden\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0membs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# batch_size x hid_size\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m         \u001b[0mscores\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moutput\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mh\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m# batch_size x num_words\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# start training\n",
    "for ITER in range (10): # CHANGE to 100\n",
    "    # training\n",
    "    random.shuffle(train_data)\n",
    "\n",
    "    model.train()\n",
    "    train_words, train_loss = 0, 0.0\n",
    "    for sent_id, sent in enumerate(train_data): # CHANGE to all train_data\n",
    "        \n",
    "        my_loss = calc_sent_loss(sent)\n",
    "        \n",
    "        train_loss += my_loss.item()\n",
    "        train_words += len(sent)\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        my_loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        if (sent_id+1) % 5000 == 0:\n",
    "            print(\"--finished %r sentences\" % (sent_id+1))\n",
    "    print(\"iter %r: train loss/word=%.4f, ppl=%.4f\" % (ITER, train_loss/train_words, math.exp(train_loss/train_words)))\n",
    "\n",
    "    # evaluation\n",
    "    model.eval()\n",
    "    dev_words, dev_loss = 0, 0.0\n",
    "    start = time.time()\n",
    "    for sent_id, sent in enumerate(val_data):\n",
    "        my_loss = calc_sent_loss(sent)\n",
    "        dev_loss += my_loss.item()\n",
    "        dev_words += len(sent)\n",
    "    print(\"iter %r: dev loss/word=%.4f, ppl=%.4f, time=%.2fs\" % (ITER, dev_loss/dev_words, math.exp(dev_loss/dev_words), time.time()-start))\n",
    "\n",
    "    # Generate a few sentences\n",
    "    for _ in range(5):\n",
    "        sent = generate_sent()\n",
    "        print(\" \".join([index_to_word[x] for x in sent]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "nlp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.13"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "154abf72fb8cc0db1aa0e7366557ff891bff86d6d75b7e5f2e68a066d591bfd7"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
